#include "../../include/DictProducer.hpp"
#include "../../include/Configuration.hpp"
#include "../../include/CppJieba.hpp"

#include <fstream>
#include <iostream>
#include <sstream>
#include <ostream>

using std::endl;
using std::cout;
using std::ifstream;
using std::ofstream;
using std::istringstream;

void DictProducer::readEnStop(const string &stopFilename)
{
    ifstream ifs;
    ifs.open(stopFilename);
    if(!ifs)
    {
        cout << "readEnStop open " << stopFilename << endl;
    }
    string lineData;
    //停用词每行一个单词
    while(std::getline(ifs, lineData))
    {
        //直接插入就行
        _stop.insert(lineData);
    }
}

//构建英文字典库
void DictProducer::buildEnDict()
{
    //读入停用词文件,不需要改变
    readEnStop("../../data/stop_words_eng.txt");
    Configuration con("../../conf/conf.conf");
    string filename = con.sendConfigure();
    con.getConf("../../data/eng", _files, "../../conf/conf.conf");
    ifstream ifs;
    ifs.open(filename);
    if(!ifs)
    {
        cout << "ifstream open " << filename << "error" << endl;
    }

    //需要将停用词先读出来放入进容器//是否需要写一个函数去读取停用词文件
    string lineData;
    size_t lineNumber = 0;//行号
    //while(std::getline(ifs,lineData))
    for(auto &it : _files)
    {
        ifstream ifs2;
        ifs2.open(it);
        while(std::getline(ifs2, lineData), ifs2.good())
        {
            //过滤改行数据,变为小写模式,分割单词
            preProcessLine(lineData);

            istringstream line(lineData);
            string word;
            while(line >>  word)
            {
                //查看是否是停用词
                auto sit = _stop.find(word);//时间复杂度O(1)
                if(sit != _stop.end())
                {
                    continue;//跳出进入下一个单词
                }
                //索引库的建立 此时需要将单词分解   索引库是否应该将停用词剔除?
                createEnIndex(word,lineNumber);

                //查找该单词是否出现
                auto it = _dict.find(word);
            
                if(it != _dict.end())//找到了
                {
                    ++_dict[word];//词频++
                }
                else
                {
                    _dict[word] = 1;//插入且词频置为1
                }
            }
            ++lineNumber;
        }
        ifs2.close();

    }
    ifs.close();
}
void DictProducer::preProcessLine(string &line)
{
    for(size_t i=0;i< line.size();++i)
    {
        if(!isalpha(line[i]))//处理单词，如果不是字母就用空格代替abc? abc123 Hello
        {
            line[i] = ' ';
        }
        else if(isupper(line[i]))//如果是大写就转为小写
        {
            line[i] = tolower(line[i]);
        }
    }
}

void DictProducer::buildCnDict()
{
    CppJieba jieba;
    _files.clear();
    _dict.clear();
    //读入停用词文件
    readEnStop("../../data/stop_words_zh.txt");
    Configuration con("../../conf/confCh.conf");
    string filename = con.sendConfigure();
    con.getConf("../../data/art", _files, "../../conf/confCh.conf");
    ifstream ifs;
    ifs.open(filename);
    if(!ifs)
    {
        cout << "ifstream open " << filename << "error" << endl;
    }
    string lineData;
    size_t lineNumber = 0;//行号
    for(auto &it : _files)
    {
        ifstream ifs2;
        ifs2.open(it);
        while(std::getline(ifs2, lineData), ifs2.good())
        {
           vector<string>tmp = jieba.cut(lineData);

           //遍历tmp看是否有与停用词重复的
           for(auto &it : tmp)
           {
               auto sit = _stop.find(it);
               if(sit != _stop.end())
               {
                   continue;
               }
               createCnIndex(it, lineNumber);

               auto it1 = _dict.find(it);
               if(it1 != _dict.end())
               {
                   ++_dict[it];//词频
               }else
               {
                   _dict[it] = 1;
               }
           }
           ++lineNumber;
        }
        ifs2.close();
    }
    ifs.close();
}

void DictProducer::createEnIndex(const string & word,int lineNum)//英文索引库
{
    for(size_t i=0; i < word.size(); ++i)
    {
        _index[word[i]].insert(lineNum);
    }
}

void DictProducer::createCnIndex(const string &word, int lineNum)
{
    string tmp = word;
    vector<string> vec;
    int size = 3;
    for(int i = 0; i < tmp.size(); i += 3)
    {
        string tmp1 = tmp.substr(i, 3);
        _chIndex[tmp1].insert(lineNum);
    }
}

void DictProducer::enStore(const string &dict, const string &index)
{
    //建立词典库
    ofstream ofs(dict);
    for(auto &it : _dict)
    {
        ofs << it.first << ' ' << it.second << endl;
    }
    ofs.close();

    ofstream ofs2(index);
    for(auto &it1 : _index)
    {
        ofs2 << it1.first << " ";
        set<int>::iterator it3 = it1.second.begin();
        for(;it3 != it1.second.end(); ++it3)
        {
            ofs2 << *it3 << " ";
        }
        ofs2 << endl;
    }
}
void DictProducer::chStore(const string &dict, const string &index)
{
    //建立词典库
    ofstream ofs(dict);
    for(auto &it : _dict)
    {
        ofs << it.first << ' ' << it.second << endl;
    }
    ofs.close();

    ofstream ofs2(index);
    for(auto &it1 : _chIndex)
    {
        ofs2 << it1.first << " ";
        set<int>::iterator it3 = it1.second.begin();
        for(;it3 != it1.second.end(); ++it3)
        {
            ofs2 << *it3 << " ";
        }
        ofs2 << endl;
    }
}
